{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Reinforcement Learning using AlphaZero methodology\n",
    "\n",
    "Please see https://applied-data.science/blog/how-to-build-your-own-alphazero-ai-using-python-and-keras/ for further notes on the codebase"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. First load the core libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "# %matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "np.set_printoptions(suppress=True)\n",
    "\n",
    "from shutil import copyfile\n",
    "import random\n",
    "\n",
    "\n",
    "from keras.utils import plot_model\n",
    "\n",
    "from game import Game, GameState\n",
    "from agent import Agent\n",
    "from memory import Memory\n",
    "from model import Residual_CNN\n",
    "from funcs import playMatches, playMatchesBetweenVersions\n",
    "\n",
    "import loggers as lg\n",
    "\n",
    "from settings import run_folder, run_archive_folder\n",
    "import initialise\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Now run this block to start the learning process\n",
    "\n",
    "This block loops for ever, continually learning from new game data.\n",
    "\n",
    "The current best model and memories are saved in the run folder so you can kill the process and restart from the last checkpoint."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lg.logger_main.info('=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*')\n",
    "lg.logger_main.info('=*=*=*=*=*=.      NEW LOG      =*=*=*=*=*')\n",
    "lg.logger_main.info('=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*')\n",
    "\n",
    "env = Game()\n",
    "\n",
    "# If loading an existing neural network, copy the config file to root\n",
    "if initialise.INITIAL_RUN_NUMBER != None:\n",
    "    copyfile(run_archive_folder + env.name + '/run' + str(initialise.INITIAL_RUN_NUMBER).zfill(4) + '/config.py', './config.py')\n",
    "\n",
    "import config\n",
    "\n",
    "######## LOAD MEMORIES IF NECESSARY ########\n",
    "\n",
    "if initialise.INITIAL_MEMORY_VERSION == None:\n",
    "    memory = Memory(config.MEMORY_SIZE)\n",
    "else:\n",
    "    print('LOADING MEMORY VERSION ' + str(initialise.INITIAL_MEMORY_VERSION) + '...')\n",
    "    memory = pickle.load( open( run_archive_folder + env.name + '/run' + str(initialise.INITIAL_RUN_NUMBER).zfill(4) + \"/memory/memory\" + str(initialise.INITIAL_MEMORY_VERSION).zfill(4) + \".p\",   \"rb\" ) )\n",
    "\n",
    "######## LOAD MODEL IF NECESSARY ########\n",
    "\n",
    "# create an untrained neural network objects from the config file\n",
    "current_NN = Residual_CNN(config.REG_CONST, config.LEARNING_RATE, (2,) + env.grid_shape,   env.action_size, config.HIDDEN_CNN_LAYERS)\n",
    "best_NN = Residual_CNN(config.REG_CONST, config.LEARNING_RATE, (2,) +  env.grid_shape,   env.action_size, config.HIDDEN_CNN_LAYERS)\n",
    "\n",
    "#If loading an existing neural netwrok, set the weights from that model\n",
    "if initialise.INITIAL_MODEL_VERSION != None:\n",
    "    best_player_version  = initialise.INITIAL_MODEL_VERSION\n",
    "    print('LOADING MODEL VERSION ' + str(initialise.INITIAL_MODEL_VERSION) + '...')\n",
    "    m_tmp = best_NN.read(env.name, initialise.INITIAL_RUN_NUMBER, best_player_version)\n",
    "    current_NN.model.set_weights(m_tmp.get_weights())\n",
    "    best_NN.model.set_weights(m_tmp.get_weights())\n",
    "#otherwise just ensure the weights on the two players are the same\n",
    "else:\n",
    "    best_player_version = 0\n",
    "    best_NN.model.set_weights(current_NN.model.get_weights())\n",
    "\n",
    "#copy the config file to the run folder\n",
    "copyfile('./config.py', run_folder + 'config.py')\n",
    "plot_model(current_NN.model, to_file=run_folder + 'models/model.png', show_shapes = True)\n",
    "\n",
    "print('\\n')\n",
    "\n",
    "######## CREATE THE PLAYERS ########\n",
    "\n",
    "current_player = Agent('current_player', env.state_size, env.action_size, config.MCTS_SIMS, config.CPUCT, current_NN)\n",
    "best_player = Agent('best_player', env.state_size, env.action_size, config.MCTS_SIMS, config.CPUCT, best_NN)\n",
    "#user_player = User('player1', env.state_size, env.action_size)\n",
    "iteration = 0\n",
    "\n",
    "while 1:\n",
    "\n",
    "    iteration += 1\n",
    "    reload(lg)\n",
    "    reload(config)\n",
    "    \n",
    "    print('ITERATION NUMBER ' + str(iteration))\n",
    "    \n",
    "    lg.logger_main.info('BEST PLAYER VERSION: %d', best_player_version)\n",
    "    print('BEST PLAYER VERSION ' + str(best_player_version))\n",
    "\n",
    "    ######## SELF PLAY ########\n",
    "    print('SELF PLAYING ' + str(config.EPISODES) + ' EPISODES...')\n",
    "    _, memory, _, _ = playMatches(best_player, best_player, config.EPISODES, lg.logger_main, turns_until_tau0 = config.TURNS_UNTIL_TAU0, memory = memory)\n",
    "    print('\\n')\n",
    "    \n",
    "    memory.clear_stmemory()\n",
    "    \n",
    "    if len(memory.ltmemory) >= config.MEMORY_SIZE:\n",
    "\n",
    "        ######## RETRAINING ########\n",
    "        print('RETRAINING...')\n",
    "        current_player.replay(memory.ltmemory)\n",
    "        print('')\n",
    "\n",
    "        if iteration % 5 == 0:\n",
    "            pickle.dump( memory, open( run_folder + \"memory/memory\" + iteration + \".p\", \"wb\" ) )\n",
    "\n",
    "        lg.logger_memory.info('====================')\n",
    "        lg.logger_memory.info('NEW MEMORIES')\n",
    "        lg.logger_memory.info('====================')\n",
    "        \n",
    "        memory_samp = random.sample(memory.ltmemory, min(1000, len(memory.ltmemory)))\n",
    "        \n",
    "        for s in memory_samp:\n",
    "            current_value, current_probs, _ = current_player.get_preds(s['state'])\n",
    "            best_value, best_probs, _ = best_player.get_preds(s['state'])\n",
    "\n",
    "            lg.logger_memory.info('MCTS VALUE FOR %s: %f', s['playerTurn'], s['value'])\n",
    "            lg.logger_memory.info('CUR PRED VALUE FOR %s: %f', s['playerTurn'], current_value)\n",
    "            lg.logger_memory.info('BES PRED VALUE FOR %s: %f', s['playerTurn'], best_value)\n",
    "            lg.logger_memory.info('THE MCTS ACTION VALUES: %s', ['%.2f' % elem for elem in s['AV']]  )\n",
    "            lg.logger_memory.info('CUR PRED ACTION VALUES: %s', ['%.2f' % elem for elem in  current_probs])\n",
    "            lg.logger_memory.info('BES PRED ACTION VALUES: %s', ['%.2f' % elem for elem in  best_probs])\n",
    "            lg.logger_memory.info('ID: %s', s['state'].id)\n",
    "            lg.logger_memory.info('INPUT TO MODEL: %s', current_player.model.convertToModelInput(s['state']))\n",
    "\n",
    "            s['state'].render(lg.logger_memory)\n",
    "            \n",
    "        ######## TOURNAMENT ########\n",
    "        print('TOURNAMENT...')\n",
    "        scores, _, points, sp_scores = playMatches(best_player, current_player, config.EVAL_EPISODES, lg.logger_tourney, turns_until_tau0 = 0, memory = None)\n",
    "        print('\\nSCORES')\n",
    "        print(scores)\n",
    "        print('\\nSTARTING PLAYER / NON-STARTING PLAYER SCORES')\n",
    "        print(sp_scores)\n",
    "        #print(points)\n",
    "\n",
    "        print('\\n\\n')\n",
    "\n",
    "        if scores['current_player'] > scores['best_player'] * config.SCORING_THRESHOLD:\n",
    "            best_player_version = best_player_version + 1\n",
    "            best_NN.model.set_weights(current_NN.model.get_weights())\n",
    "            best_NN.write(env.name, best_player_version)\n",
    "\n",
    "    else:\n",
    "        print('MEMORY SIZE: ' + str(len(memory.ltmemory)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The following panels are not involved in the learning process\n",
    "\n",
    "### Play matches between versions (use -1 for human player)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from game import Game\n",
    "from funcs import playMatchesBetweenVersions\n",
    "import loggers as lg\n",
    "\n",
    "env = Game()\n",
    "playMatchesBetweenVersions(env, 1, 1, 1, 10, lg.logger_tourney, 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pass a particular game state through the neural network (setup below for Connect4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs = GameState(np.array([\n",
    "    0,0,0,0,0,0,0,\n",
    "    0,0,0,0,0,0,0,\n",
    "    0,0,0,0,0,0,0,\n",
    "    0,0,0,0,0,0,0,\n",
    "    0,0,0,0,0,0,0,\n",
    "    0,0,0,0,0,0,0\n",
    "]), 1)\n",
    "\n",
    "preds = current_player.get_preds(gs)\n",
    "\n",
    "print(preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### See the layers of the current neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_player.model.viewLayers()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Output a diagram of the neural network architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import plot_model\n",
    "plot_model(current_NN.model, to_file=run_folder + 'models/model.png', show_shapes = True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "deepreinforcement",
   "language": "python",
   "name": "deepreinforcement"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
